# -*- coding: utf-8 -*-
import scrapy
from webspider.items import LagouspiderItem


class LagouSpider(scrapy.Spider):
    name = 'lagou'
    allowed_domains = ['lagou.com']
    start_urls = ['https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=']

    custom_settings = {
        "COOKIES_ENABLED": False,
        'DEFAULT_REQUEST_HEADERS': {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7',
            'Connection': 'keep-alive',
            'Cookie': 'user_trace_token=20200319174132-115e0be2-442e-489d-90b9-de4729f05e9d; LGUID=20200319174134-3202f3df-41ba-491b-af44-5e850cc93f1b; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22170f22b649213f-077dd264c6de31-4313f6a-2073600-170f22b6493318%22%2C%22%24device_id%22%3A%22170f22b649213f-077dd264c6de31-4313f6a-2073600-170f22b6493318%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; _ga=GA1.2.675436888.1584610895; X_MIDDLE_TOKEN=ea11fff4da71ef81389d42496204612b; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1584610895,1585655558; _gid=GA1.2.2075645069.1585655580; LG_HAS_LOGIN=1; privacyPolicyPopup=false; index_location_city=%E4%B8%8A%E6%B5%B7; LGSID=20200401014606-92655490-5463-4450-94ce-a8fe1eeb79c3; X_HTTP_TOKEN=4726ce100c00d4b812867658517979651b4500ce8f; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1585676822; LGRID=20200401023203-fe1ec01d-4aa9-45b2-82ef-efc48e0f608a; gate_login_token=fdf2d73bcbff1d97580c3393cd1c73b895d0073bb412932dda39cbba0d100de0; LG_LOGIN_USER_ID=89e4860764f5a5cbf9f987d9e8e2359282630d968fb341fed8dbee860a30ac1c; _putrc=88E395D1E285BD49123F89F2B170EADC; login=true; unick=%E9%A9%AC%E6%98%8C%E9%A1%BA',
            'Host': 'www.lagou.com',
            'Origin': 'https://www.lagou.com',
            'Referer': 'https://www.lagou.com/',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
        }
    }

    def parse(self, response):
        job_list = response.xpath("//*[@id='s_position_list']//ul/li")
        for job in job_list:
            item = LagouspiderItem()
            # item['url'] = response.url
            item['name'] = job.xpath("//h3/text()")
            yield item